package com.chance.cc.crawler.development.bootstrap.pcauto;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;

/**
 * 文章采集
 */
public class PcAutoArticleStart {

    private static final String domain = "pcauto";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }


    public static void main(String[] args) {
        doArticleCrawler();
    }

    public static void doArticleCrawler(){
        String site = "article";
        String url = "https://www.pcauto.com.cn/nation/2576/25767151.html";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .domain(domain)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.key)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1624291200000L,System.currentTimeMillis()}))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .proxy(proxy)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domain,domain + "_trigger",System.currentTimeMillis(),domain+ "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\data\\pcauto\\pcauto_forum_reply_test.json",false)
                .requestRecord(requestRecord)
//                .crawlerThreadNum(3)
                .build("com.chance.cc.crawler.development.scripts.pcauto")
                .start();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
    }

}
